#!/usr/bin/env python
from lib.format_time import *
from lib.HandleMysql import *
from lib.handle_mongo import *
from lib.switch_code import *
from lib.load_data import *
from zs_conn import *
from config_tools import *
import urllib
import base64
import sys
import re
import time
import datetime
reload(sys)
sys.setdefaultencoding('utf-8')

def get_click_info(date):
    
    dict_conn = set_dict_conn()
    stat_conn = set_stats_conn()
    stat_table = "zs_click_info"
    line_list = []
    user_ip_dict = get_user_ip()
    for i in range(2):
        url = "ftp://jz03.service.58dns.org/adserver/zhaoshang/zhaoshang.log.%s"%date
        if i == 1:    
            url = "ftp://jz04.service.58dns.org/adserver/zhaoshang/zhaoshang.log.%s"%date
            
        file_path = "/work3/opt/analytic/lh_tmp_data/zhaoshang/click_info_%s_%s.txt"%(date,i)
            
        #print url,file_path    
        urllib.urlretrieve(url, file_path)                       
        
        f = open(file_path)
        for line in f.readlines():
            arr = line.split(' ')
            #print line
            user_id = arr[1]
            year = arr[2]
            if len(year) == 8:
                year = "20%s"%year
            date_str = arr[3]
            click_date = '%s %s'%(year,date_str)
            
            channel = arr[4]
            
            params = arr[6].replace('\n','')
            #print params
            p = params.split('$')
            refer = p[0].replace('from=','')
            target_url= p[1].replace('target_url=','')
            ad_id= p[3].replace('id=','')
            user_ip= p[4].replace('ip=','')
            pc= p[5].replace('pc=','')
            cateid = 0
            try:
                cateid= p[6].replace('cateid=','')
            except Exception:
                continue
            cooperator = 0
            try:
                cooperator= p[7].replace('cooperator=','')
            except Exception:
                continue
            
            key = "%s_%s"%(user_ip,cooperator)
            if user_ip_dict.has_key(key):
                t = user_ip_dict[key]
                if t == "0":
                    user_ip_dict[key] = "1"
                else:
                    continue                    
            
            position = 0
            try:
                position= p[8].replace('position=','')
            except Exception:
                continue
            state = 1
            if pc.isdigit() == False:
                state = 0
                pc = 1
                
            if cateid.isdigit() == False:
                state = 0
                cateid = 0
                
            if cooperator.isdigit() == False:
                state = 0
            log_info =' '  
            if state == 0:
                log_info = line.replace('\n','')
            if refer =="null":
                continue
                #state = 2
                #log_info = line.replace('\n','')
            
            #city
            city_id = 0
            city_name = ''
            list_name = refer.split('.')[0].replace('http://','')
            q = "select displocalid,localname from t_displocal where listname='%s'"%list_name
            its = dict_conn.get_results(q)
            if len(its)>0:
                city_id = its[0][0]
                city_name = its[0][1]
                
            dc = None
            if cateid > 0 :
                if position=='5_1' or position=='5_2' or position == '7' or position == '9':
                    dc = get_gs_cates(cateid,dict_conn)
                else:
                    dc = get_cates(cateid,dict_conn)    
            cat1 = 0
            cat1name = ''
            cat2 = 0
            cat2name=''
            cat3 = 0
            cat3name=''
            if dc != None:
                if len(dc) > 0:
                    cat1 = dc[0][0]
                    cat1name = dc[0][1]
                if len(dc) > 1:
                    cat2 = dc[1][0]
                    cat2name=dc[1][1]
                if len(dc) > 2:
                    cat3 = dc[2][0]
                    cat3name=dc[2][1]
            
            param = [date,user_id,click_date,channel,refer,target_url,ad_id,user_ip,pc,city_id,city_name,cat1,cat1name,cat2,cat2name,cat3,cat3name,cooperator,position,state,log_info]
            line_list.append('\t'.join(map(lambda i : str(i), param)))
        
        remove_file(file_path)
    cont = '\n'.join(line_list)
    path = "/work3/opt/analytic/lh_tmp_data/zhaoshang/click/"
    file_name = "click_info_%s.txt"%date_str
    writeTextToFile(path,file_name,"w",cont)
    
    load_data(stat_conn,path+file_name,stat_table,'\t')
    #remove_file(path+file_name)
    
    q = "select cooperator,user_ip,count(1) from zs_click_info where stat_date='%s' group by cooperator,user_ip having(count(1))>40"%date
    items = stat_conn.get_results(q)
    for item in items:
        coop = item[0]
        user_ip = item[1]
        #count = item[2]
        
        sql = "insert into zs_user_ip(user_ip,coop) values ('%s','%s');"%(user_ip,coop)
        #print sql
        stat_conn.execute_sql(sql)
        sql = "delete from zs_click_info where user_ip='%s' and stat_date='%s' and cooperator='%s'"%(user_ip,date,coop)
        stat_conn.execute_sql(sql)
    
    stat_conn.close()
    dict_conn.close()


def get_click_info_from_tomcat(date):
    stat_conn = set_stats_conn()
    stat_table = "zs_click_info_tomcat"
    line_list = []
    user_ip_dict = get_user_ip()
    
    reg_str = "\[(.*)\].*ad[J|j]ump\?(.*)&position=([0-9](_[0-9])?).*'((?:(?:25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d)))\.){3}(?:25[0-5]|2[0-4]\d|((1\d{2})|([1-9]?\d))))'"
    pattern =  re.compile(reg_str)
    for i in range(2):
        url = "ftp://jz03.service.58dns.org/localhost_access_log.%s.txt"%date
        if i == 1:    
            url = "ftp://jz04.service.58dns.org/localhost_access_log.%s.txt"%date
            
        file_path = "/work3/opt/analytic/lh_tmp_data/zhaoshang/click_info_tomcat_%s_%s.txt"%(date,i)
        #print url,file_path    
        urllib.urlretrieve(url, file_path)                       
        
        f = open(file_path)
        for line in f.readlines():
            #print line.find('cooperator')
            if line.find('cooperator') == -1:
                continue
            
            params = ''
            position = ''
            user_ip = ''
            click_time = ''
            matchs = pattern.search(line)
            try:
                click_time = matchs.group(1)
                params = matchs.group(2)
                position = matchs.group(3)
                user_ip = matchs.group(5)
            except Exception,e:
                #print line
                continue
            
            #print click_time
            arr = params.split("&")
            target = arr[0].replace('target=','')
            refer = params.replace("%s&from="%arr[0],'')
            refer = "%s&positon=%s"%(refer,position)
            
            
            pc = 1
            cate_id = 0
            cooperator = 0
            p_arr = refer.split("&")
            for s in p_arr:
                if s.find("pc=")>-1:
                    pc = s.replace("pc=",'')
                elif s.find("cateid=")>-1:
                    cate_id = s.replace("cateid=",'')
                elif s.find("cooperator=")>-1:
                    cooperator = s.replace("cooperator=",'')
                    
            if refer.find("http")==-1:
                continue
                #refer = 'null'
                #print refer
            key = "%s_%s"%(user_ip,cooperator)
            if user_ip_dict.has_key(key):
                t = user_ip_dict[key]
                if t == "0":
                    user_ip_dict[key] = "1"
                else:
                    continue  
                
            click_time = str(datetime.datetime.strptime(click_time,'%d/%b/%Y:%H:%M:%S +0800'))
            param = [date,user_ip,click_time,cooperator,cate_id,pc,position,refer,target]
            line_list.append('\t'.join(map(lambda i : str(i), param)))
        remove_file(file_path)
        
    cont = '\n'.join(line_list)
    path = "/work3/opt/analytic/lh_tmp_data/zhaoshang/click/"
    file_name = "tomcat_click_info_%s.txt"%date.replace('-','')
    writeTextToFile(path,file_name,"w",cont)
    
    load_data(stat_conn,path+file_name,stat_table,'\t')
    #remove_file(path+file_name)
    
    stat_conn.close()
            
            

def get_user_ip():
    user_ip_dict = {}
    stat_conn = set_stats_conn()
    q = "select user_ip,coop from zs_user_ip"
    items = stat_conn.get_results(q)
    for item in items:
        user_ip = item[0]
        coop = item[1]
        
        key = "%s_%s"%(user_ip,coop)
        user_ip_dict[key] = "0"
    stat_conn.close()
    return user_ip_dict


def execute(date):
    #get_click_info(date)
    try:
        pass
        get_click_info(date)
    except Exception,e:
        print 'get_click_info:',e
    
    try:
        pass
        get_click_info_from_tomcat(date)
    except Exception,e:
        print 'get_click_info_from_tomcat:',e

def main():
   if len(sys.argv)==1:
        start_date=get_yesterday()
        end_date=start_date
   elif len(sys.argv)==2:
        start_date=format_date(sys.argv[1])
        end_date=start_date
   elif len(sys.argv)==3:
        start_date=format_date(sys.argv[1])
        end_date=format_date(sys.argv[2])
   stat_date=start_date
   
   while stat_date>=start_date and stat_date<=end_date:
        print "The program of the date of %s is beginning at %s " %(stat_date,get_now())
        execute( stat_date)
        #get_click_info_from_tomcat(stat_date)
        print "The program of the date of %s is end at %s " %(stat_date,get_now())
        stat_date=var_date(stat_date)

if __name__=='__main__':
    main()
